package com.levy.gameTest;

import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson.JSONObject;
import com.levy.Adv;
import com.levy.utils.FileUtils;
import org.apache.log4j.Logger;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.junit.Test;
import org.nutz.mvc.annotation.AdaptBy;

import java.io.*;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.*;

public class GameCrawler {
    public static final Logger log = Logger.getLogger(GameCrawler.class);


    private static boolean isValiableHostCommonGame(String s) {
        boolean flag = s.contains("Host:")
                && !s.contains("icloud-content.com")
                && !s.contains("icloud.com")
                && !s.contains("apple.com")
                && !s.contains("doubleclick.net")
                && !s.contains("apple.com")
                && !s.contains("google")
                && !s.contains("myapp.com")
//                && !s.contains("googleusercontent.com")
//                && !s.contains("googleadservices.com")
//                && !s.contains("googlesyndication.com")
//                && !s.contains("googleapis.com")
//                && !s.contains("googlesyndication.com")
//                && !s.contains("google-analytics.com")
                && !s.contains("youtube")
                && !s.contains("mzstatic.com")
                && !s.contains("facebook.com")
                && !s.contains("jquery.com")

                && !s.contains("xiangji.qq.com")
                && !s.contains("jiguang")
                && !s.contains("ixigua.com")
                && !s.contains("umeng.com")
                && !s.contains("qiniucdn.com")
                && !s.contains("baidu.com")
                && !s.contains("igexin.com")
                && !s.contains("mi.com")
                && !s.contains("mi.net")
                && !s.contains("microsoft.com")
                && !s.contains("sougou.com")
                && !s.contains("weixin")
                && !s.contains("amap.com")
                && !s.contains("google.com")
                && !s.contains("sogou.com")
//                && !s.contains("taobao.com")
                && !s.contains("2345.com")
                && !s.contains("360.cn")
                && !s.contains("localhost")
                && !s.contains("oupeng.com")
                && !s.contains("csdn.net")
                && !s.contains("360safe.com")
                && !s.contains("unity3d.com")
                && !s.contains("daxiangdaili.com")
                && !s.contains("mee.gov.cn")
                && !s.contains("miui.com")
                && !s.contains("ylxqgo.com")
                && !s.contains("i.gtimg.cn")
                && !s.contains("jpush.cn")
                && !s.contains("qq.com");
        return flag;
    }


    private static boolean isValiableHostByTencntGame(String s) {
        boolean
                flag = s.contains("Host:")
                && !s.contains("icloud-content.com")
                && !s.contains("icloud.com")
                && !s.contains("apple.com")
                && !s.contains("doubleclick.net")
                && !s.contains("apple.com")
                && !s.contains("google")
//                && !s.contains("googleusercontent.com")
//                && !s.contains("googleadservices.com")
//                && !s.contains("googlesyndication.com")
//                && !s.contains("googleapis.com")
//                && !s.contains("googlesyndication.com")
//                && !s.contains("google-analytics.com")
                && !s.contains("youtube")
                && !s.contains("mzstatic.com")
                && !s.contains("facebook.com")
                && !s.contains("jquery.com")

                && !s.contains("xiangji.qq.com")
                && !s.contains("jiguang")
                && !s.contains("ixigua.com")
                && !s.contains("umeng.com")
                && !s.contains("qiniucdn.com")
                && !s.contains("igexin.com")
                && !s.contains("baidu.com")
                && !s.contains("mi.com")
                && !s.contains("mi.net")
                && !s.contains("microsoft.com")
                && !s.contains("sougou.com")
                && !s.contains("weixin")
                && !s.contains("amap.com")
                && !s.contains("google.com")
                && !s.contains("sogou.com")
                && !s.contains("taobao.com")
                && !s.contains("2345.com")
                && !s.contains("360.cn")
                && !s.contains("localhost")
                && !s.contains("oupeng.com")
                && !s.contains("csdn.net")
                && !s.contains("ylxqgo.com")
                && !s.contains("360safe.com")
                && !s.contains("unity3d.com")
                && !s.contains("daxiangdaili.com")
                && !s.contains("mee.gov.cn")
                && !s.contains("jpush.cn")
                && !s.contains("unipay.qq.com")
                && !s.contains("connect.qq.com")
                && !s.contains("i.gtimg.cn")
                && !s.contains("miui.com");

        return flag;
    }


    private static void merginToExcel(String dir) throws IOException {
//        ArrayList<String> fileList = FileUtils.getFiles(dir);
        List<String> allCoverFileList = new ArrayList<>();
        File file = new File(dir);
        File[] files = file.listFiles();
        for (File ff : files) {
            if (ff.isDirectory() && ff.getName().contains("转换")) {
                allCoverFileList.add(ff.getAbsolutePath());
            }
        }


        SXSSFWorkbook wb = null;
        Sheet sheet = null;
        try {
            wb = new SXSSFWorkbook();
        } catch (Exception e) {
            e.printStackTrace();
        }

        for (String filePath : allCoverFileList) {

            File fff = new File(filePath);
            File[] files1 = fff.listFiles();
            int i = filePath.lastIndexOf("\\");
            String parentName = filePath.substring(i + 1, filePath.length());

            sheet = wb.createSheet(parentName.replace("_转换", ""));

            sheet.setDefaultColumnWidth(30);

            Row row = sheet.createRow(0);
            row.createCell(0).setCellValue("游戏");
            row.createCell(1).setCellValue("HOST");
            int index = 1;
            for (File f : files1) {
                String absolutePath = f.getAbsolutePath();
                if (absolutePath.contains(".txt")) {
                    String name = f.getName().replace(".txt", "");
                    Scanner sc = new Scanner(new FileInputStream(f));
                    StringBuilder sb = new StringBuilder();
                    while (sc.hasNext()) {
                        String s1 = sc.nextLine();
                        sb.append(s1.replace("Host: ", "")).append("\n");
                    }
                    row = sheet.createRow(index++);
                    row.createCell(0).setCellValue(name);
                    row.createCell(1).setCellValue(sb.toString());
                    System.out.println("+++" + parentName + "_" + name + "_over +++" + index);
                }

            }


        }

        DateFormat df = new SimpleDateFormat("yyyyMMddHHmmss");
        String today = df.format(new Date());
        String fpath = dir + "//转换完HOST数据_" + today + ".xls";
        OutputStream out = new FileOutputStream(new File(fpath));
        wb.write(out);
        out.flush();
        out.close();
        System.out.println("merginToExcel Over");

    }


    private static String getHostFromFile(String filePath) throws FileNotFoundException {
        Set<String> set = new HashSet<>();
        File file = new File(filePath);
        Scanner sc = new Scanner(new FileInputStream(file));
        if (filePath.contains("腾讯")) {
            while (sc.hasNext()) {
                String s = sc.nextLine();
                if (isValiableHostByTencntGame(s)) {
                    set.add(s);
                }
            }
        } else {
            while (sc.hasNext()) {
                String s = sc.nextLine();
                if (isValiableHostCommonGame(s)) {
                    set.add(s);
                }

            }
        }

        Set<String> sortSet = new TreeSet<String>(new Comparator<String>() {
            @Override
            public int compare(String o1, String o2) {
                return o1.compareTo(o2);
            }
        });
        sortSet.addAll(set);

        StringBuilder sb = new StringBuilder();
        if (sortSet.size() > 0) {
            for (String s : sortSet) {
                sb.append(s).append("\r\n");
            }

        }


        return sb.toString();
    }


    private static void parseTxt(String dir) throws Exception {
        ArrayList<String> fileList = FileUtils.getFiles(dir);
        String parent = "";
        for (String file : fileList) {
            File file1 = new File(file);
            parent = file1.getParent();
            file = file.trim();
            if (file != null && file != "") {
                String ff = file.substring(file.lastIndexOf("\\") + 1, file.length() - 4);
                String hostFromFile = getHostFromFile(file);
                FileUtils.write(parent + "_转换/" + ff + ".txt", hostFromFile, "utf-8");

                System.out.println(file + "_over");
            }
        }
    }


    public static void deleteAllFilesOfDir(File path) {
        if (null != path) {
            if (!path.exists()) {
                return;
            }
            if (path.isFile()) {
                boolean result = path.delete();
                int tryCount = 0;
                while (!result && tryCount++ < 10) {
                    System.gc(); // 回收资源
                    result = path.delete();
                }
            }
            File[] files = path.listFiles();
            if (null != files) {
                for (int i = 0; i < files.length; i++) {
                    deleteAllFilesOfDir(files[i]);
                }
            }
            path.delete();
        }
    }

    @Test
    public void test8() throws IOException {
        String path = "G://11//";
        String savePath = path+"2.txt";
        File rootFile = new File(path);
        String[] fileDirNameList = rootFile.list();
        System.out.println(fileDirNameList.length);
        List<Adv> list = new ArrayList<>();
        for (String dirName : fileDirNameList) {
            String sPath = path + dirName;
            File dirFile = new File(sPath);
            if (dirFile.isDirectory()) {
                //目录下的文件
                String[] detailFileNameList = dirFile.list();
                List<String> allStr = new ArrayList<>();
                for (String detailFileName : detailFileNameList) {
                    String ssPath = sPath + "\\" + detailFileName;
                    List<String> strings = FileUtil.readUtf8Lines(ssPath);
                    for (String s : strings) {
                        Adv adv = new Adv();
                        adv.setName(detailFileName.replaceAll(".txt", ""));
                        adv.setUrl(s.replaceAll("Host:", ""));
                        adv.setUrlBig("");
                        list.add(adv);
                    }
                }
                wirteAdvList2ToTxtFile(list, savePath);
            }
        }

    }

    private void wirteAdvList2ToTxtFile(List<Adv> list, String path) {
        StringBuilder sb = new StringBuilder();
        for (Adv adv : list) {
            sb.append(adv.getName()).append(",")
                    .append(adv.getUrl()).append(",")
                    .append(",").append("\r\n");
        }
        FileUtil.writeUtf8String(sb.toString(), path);
    }


    @Test
    public void test7() throws IOException {
        //有米数据集 youmi
        String path = "G://adv//";
        File rootFile = new File(path);
        String[] fileDirNameList = rootFile.list();
        System.out.println(fileDirNameList.length);
        for (String dirName : fileDirNameList) {
            String sPath = path + dirName;
            File dirFile = new File(sPath);
            if (dirFile.isDirectory()) {
                //目录下的文件
                String[] detailFileNameList = dirFile.list();
                List<String> allStr = new ArrayList<>();
                for (String detailFileName : detailFileNameList) {
                    String ssPath = sPath + "\\" + detailFileName;
                    List<String> strings = FileUtil.readUtf8Lines(ssPath);
                    allStr.addAll(strings);
                }
                Set<String> set = getDataDetailSetList(allStr);
                List<Adv> advs = covAdvList(set);
                wirteAdvListToTxtFile(path, dirName, advs);
            }
        }
    }

    private List<Adv> covAdvList(Set<String> set) {
        List<Adv> advList = new ArrayList<>();
        for (String textStr : set) {
            try {
                if (StrUtil.isNotBlank(textStr)) {
                    Adv adv = JSONObject.parseObject(textStr, Adv.class);
                    String url = adv.getUrl();
                    if (StrUtil.isNotBlank(url)) {
                        String s = url.replaceAll("https://", "").replaceAll("http://", "");
                        int i = s.indexOf("/");
                        String domain = s.substring(0, i);
                        adv.setDomain(domain);
                        String proPath = s.substring(i, s.length());
                        adv.setProPath(proPath);
                        advList.add(adv);
                    }
                }


            } catch (Exception e) {
                e.printStackTrace();
            }

        }

        return advList;
    }

    private Set<String> getDataDetailSetList(List<String> detailFileName) {
        Set<String> set = new TreeSet<>();
        for (String str : detailFileName) {
            set.add(str.substring(1, str.length()));
        }
        return set;
    }

    private void wirteAdvListToTxtFile(String path, String dirName, List<Adv> advs) {
        Set<String> set = new TreeSet<>();
        for (Adv adv : advs) {
            String s = dirName + "," + adv.getDomain() + "," + adv.getProPath() + ",";
            set.add(s);
        }

        FileUtil.writeUtf8Lines(set, path + dirName + ".txt");
        System.out.println("---" + dirName + "---over");
    }


    public static void main(String[] args) {
        try {

//            new GameCrawler().execute();

            //该目录的下一级  为游戏分类文件夹
            String dir = "G:\\11";
            File file = new File(dir);
            File[] files = file.listFiles();
            List<String> list2 = new ArrayList<>();
            for (File ff : files) {
                if (ff.isDirectory()) {
                    list2.add(ff.getAbsolutePath());
                }
            }
            for (String dd : list2) {
                System.out.println("---------" + dd + "开始处理---------");
                parseTxt(dd);
                System.out.println("---------" + dd + "处理完成---------");
            }

            merginToExcel(dir);


            //转换成功后 删除  "_转换"结尾的  文件夹
            for (File ff : files) {
                if (ff.isDirectory()) {
                    String s = ff.getAbsolutePath() + "_转换";
                    File gg = new File(s);
                    deleteAllFilesOfDir(gg);
                }
            }

            System.out.println("all over");


        } catch (Exception e) {
            e.printStackTrace();
        }

    }


}
